package com.onpositive.mediawiki.crawler;

import java.io.PrintStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import com.onpositive.knowledge.model.IThing;
import com.onpositive.mediawiki.dataminer.IReferencesStorage;
import com.onpositive.mediawiki.propertyparser.ParsedDocument;
import com.onpositive.mediawiki.propertyparser.PropertyCharacteristic;
import com.onpositive.mediawiki.propertyparser.StringValueParser;
import com.onpositive.semantic.model.api.property.IProperty;
import com.onpositive.units.AbstractValueParser;
import com.onpositive.units.ParsedDate;
import com.onpositive.units.ParsedDimension;
import com.onpositive.units.ParsedFileList;
import com.onpositive.units.ParsedIdent;
import com.onpositive.units.ParsedLinkArray;
import com.onpositive.units.ParsedProperty;
import com.onpositive.units.ParsedScalar;
import com.onpositive.units.ParsedSection;
import com.onpositive.units.ParsedSentence;
import com.onpositive.units.ParsedString;
import com.onpositive.units.ParsedValue;
import com.onpositive.units.ParsedValueSequence;
import com.onpositive.units.ParsedValueSequenceV2;
import com.onpositive.units.TextContent;
import com.onpositive.units.ValueCluster;

import storage.v1.DBStorage;
import storage.v1.DefaultParserProvider;
import storage.v1.IParserProvider;
import storage.v1.IValueExtractor;
import storage.v1.LowLevelStorage;
import storage.v1.ParserV2;
import storage.v1.ProperyDataExtractor;
import storage.v1.ReplacementRequest;
import storage.v1.ValuesExtractorV1;
import storage.v1.WikiStructureExtractor;

public class StreamExtractor extends LowLevelStorage{

	private static final long serialVersionUID = 661588476686610414L;
	
	
	private static final String[] valIdArray = new String[]{
		ParsedFileList.ID,
		ParsedLinkArray.ID,
		ParsedSection.ID,
		ParsedIdent.ID,
		ParsedString.ID,
		ParsedDate.ID,
		ParsedScalar.ID,
		ParsedDimension.ID,
		ParsedValueSequence.ID,
		ParsedValueSequenceV2.ID,
		ParsedSentence.ID,
		ValueCluster.ID,
		TextContent.ID		
	};
	
	private PrintStream pStream;

	private IParserProvider parserProvider;
	
	private boolean toStringOnly = false;

	public StreamExtractor(PrintStream pStream) {
		super(null);
		this.pStream = pStream;
		this.parser.setExtractors(new IValueExtractor[]{ new ValuesExtractorV1(), new WikiStructureExtractor() });		
	}
	
	public void setUseTemplates(boolean val){
		HashMap<String,Object> params = new HashMap<String, Object>();
		params.put("use-templates", val);
		parser.configure(params);
	}
	
	public void setExtractPictureNamesAndComments(boolean val){
		HashMap<String,Object> params = new HashMap<String, Object>();
		params.put("extract-picture-names-and-comments", val);
		parser.configure(params);
	}
	
	public void setToStringOnlyMode(boolean val) {
		
		this.toStringOnly = val;
		if(val){
			this.parserProvider = new IParserProvider() {
				
				StringValueParser stringValueParser = new StringValueParser();
				
				@Override
				public AbstractValueParser getParser(String parserId) {
					if(ParsedString.ID.equals(parserId)) {					
						return stringValueParser;
					}
					return DefaultParserProvider.getInstance().getParser(parserId);
				}
			};
			
			HashMap<String,Object> params = new HashMap<String, Object>();
			params.put("parse-strings-as-idents", true);
			parser.configure(params);
		}
		else{
			this.parserProvider = DefaultParserProvider.getInstance();
			
			HashMap<String,Object> params = new HashMap<String, Object>();
			params.put("parse-strings-as-idents", false);
			parser.configure(params);			
		}
	}
	
	public void extractAll(ProperyDataExtractor pde,boolean doFlush,boolean toString) {	
		
		if(!(references instanceof DBStorage))
			return;
		
		setToStringOnlyMode(toString);
		setExtractPictureNamesAndComments(false);
		
		DBStorage dbStorage = (DBStorage) references;			
		int size = dbStorage.size();		
		ReplacementRequest request = new ReplacementRequest();
		
		ExecutorService executor = Executors.newFixedThreadPool(4);
		for(int i = 0 ; i < size ; i++){
			if((i%1000)==0){
				Date date = new Date(System.currentTimeMillis());
				SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss");
				System.out.println( sdf.format(date)+" " + i);
			}
			
			String title = dbStorage.getTitle(i);
			ParsedDocument pd = reparseDocument(title, request, 0);
			if(doFlush)
				flushDocument(pd);
				
			if(pde!=null)
				pde.processDocument(pd);
			
			getCategory().clearDocuments();
		}		
	}

	public void parseCategory(String inputCategory, ReplacementRequest request,
			HashSet<String> toIgnore, int flags) {

		HashSet<String> processedDocuments = new HashSet<String>();
		ArrayList<String> buffer = new ArrayList<String>();

		String redirectedCategoryName = inputCategory;
		buffer.add(redirectedCategoryName);


		for (int i = 0; i < buffer.size(); i++) {

			String documentName = buffer.get(i);
			String redirected = ParserV2.redirect(documentName);
			
			if(redirected==null)
				redirected = documentName;
			
			if(toIgnore!=null&&toIgnore.contains(documentName))
				continue;
			
			if (processedDocuments.contains(documentName))
				continue;
			
			if(toIgnore!=null&&toIgnore.contains(redirected))
				continue;
			
			if (processedDocuments.contains(redirected))
				continue;
			
			Collection<String> children1 = references.getChildren(documentName);
			Collection<String> children2 = references.getChildren(documentName);
			documentName=redirected;					
			ParsedDocument pd = reparseDocument(documentName, request, flags);
			getCategory().clearDocuments();
			flushDocument(pd);

			HashSet<String> children = new HashSet<String>();
			children.addAll(children1);
			children.addAll(children2);
			for (String str : children1) {
				
				if (processedDocuments.contains(str))
					continue;

				buffer.add(str);				
			}

			processedDocuments.add(documentName);
		}
		references.saveStorage();
	}

	private void flushDocument(ParsedDocument pDoc) {
		
		String docName = pDoc.getName(); 
		HashMap<String, ParsedProperty> props = pDoc.getProps();
		if(props.isEmpty()){
			pStream.println("dn=" + docName);
			return;
		}
		for(Map.Entry<String,ParsedProperty> entry : props.entrySet()){
			String propName = entry.getKey();
			ParsedProperty prop = entry.getValue();
			LinkedHashSet<ParsedValue> values = prop.getValues();
			if(values.isEmpty()){
				pStream.println("dn=" + docName + "|pn="+propName);
				continue;
			}
			for(ParsedValue pv : values){
//				if(pv instanceof ParsedValueSequenceV2){
//					ParsedValueSequenceV2 pvs = (ParsedValueSequenceV2) pv;
//					for(ParsedValue pVal : pvs.getValuesArray()){
//						String valueString = pVal.getValueAsString();
//						pStream.println("docName=" + docName + "|propName="+propName+"|value="+valueString);
//					}
//				}
//				else{
					String valueString = pv.getValueAsString();
					pStream.println("dn=" + docName + "|pn="+propName+"|"+valueString);
//				}
			}
		}		
	}
	
	@Override
	public String getParserId(String propertyName) {
		if(toStringOnly)
			return ParsedString.ID;
		else{
			return super.getParserId(propertyName);
		}
	}
	
	protected void processTableInfo()
	{
//		if( tableInfoList.size() == 0 )
//			return ;
//		try{
//			( new StreamTableInfoProcessor( this ) ).processTableInfo() ;			
//		}
//		finally{
//			tableInfoList.clear() ;
//		}
	}

	@Override
	protected List<String> getDocumentList() {
		return null;
	}

	@Override
	protected void addSpecialProperties(Collection<IProperty> col) {
	}

	@Override
	protected IProperty getSpecialProperty(String propertyName) {
		return null;
	}

	@Override
	protected IProperty createProperty(PropertyCharacteristic pChar) {
		return null;
	}

	@Override
	protected IThing createThing(Object name) {
		return null;
	}

	public IParserProvider getParserProvider() {
		return parserProvider;
	}
	
	public IReferencesStorage getDbStorage(){
		return super.references;
	}

}
